from __future__ import print_function
import pandas as pd
import numpy as np
from sklearn import datasets
from sklearn.model_selection import train_test_split
from sklearn.model_selection import GridSearchCV
##from sklearn.metrics import classification_report
from sklearn.svm import SVC
import xgboost as xgb
from xgboost.sklearn import XGBClassifier
from sklearn import cross_validation, metrics   #Additional scklearn functions
import xlrd
import csv

#all month is indicated in six numbers(four numbers of year and two number of day)
#like '201603','201711'
#get the name of last month
def last_month(date):
    date = str(date)
    year = date[:4]
    #print year;
    month = date[4:]
    if month=='10':
        month = '09'
    elif month == '01':
        year = str(int(year)-1)
        month = '12'
    else:
        month = int(month)-1
        if month<10:
            month = '0'+str(month)
        else:
            month = str(month)
    ret = year+month
    return ret

#get the name of next month
def next_month(date):
    date = str(date)
    year = date[:4]
    #print year;
    month = date[4:]
    #print month
    if month=='09':
        month = '10'
    elif month == '12':
        year = str(int(year)+1)
        month = '01'
    else:
        month = int(month)+1
        if month<10:
            month = '0'+str(month)
        else:
            month = str(month)
    ret = year+month
    return ret

#get the file name of the training data for 12 months
def get_train(date):
    date = str(date)
    year = date[:4]
    month = date[4:]
    first = str(int(year)-1)+month
    second = last_month(date)
    ret = first+'-'+second+'_train.csv'
    return ret

#get the file name of the training data for 6 months
def get_train2(date):
    date = str(date)
    year = date[:4]
    month = date[4:]
    a = int(month)-6
    if a<1:
        first = str(int(year)-1)
        if a+12>=10:
            first = first+str(a+12)
        else:
            first = first+'0'+str(a+12)
    else:
        first = year+'0'+str(a)
    second = last_month(date)
    ret = first+'-'+second+'_train.csv'
    return ret

#get the file name of the test data
def get_test(date):
    date = str(date)
    ret = date+'pred.csv'
    return ret

#get the file name of the probability
def get_prob(date):
    date = str(date)
    ret = date+'prob.csv'
    return ret

#fit the data and write out
def modelfit(alg, ids,dtrain, dtest, predictors, writer, useTrainCV=True, cv_folds=5, early_stopping_rounds=50):   
    if useTrainCV:
        xgb_param = alg.get_xgb_params()
        xgtrain = xgb.DMatrix(dtrain[predictors].values, label=dtrain[target].values)
        xgtest = xgb.DMatrix(dtest[predictors].values)
        cvresult = xgb.cv(xgb_param, xgtrain, num_boost_round=alg.get_params()['n_estimators'], nfold=cv_folds,
            metrics='auc', early_stopping_rounds=early_stopping_rounds)
        alg.set_params(n_estimators=cvresult.shape[0])
    
    #Fit the algorithm on the data
    #print(dtrain)
    alg.fit(dtrain[predictors], dtrain['yield_class'],eval_metric='auc')
    
    #Predict training set:
    dtrain_predictions = alg.predict(dtrain[predictors])
    dtrain_predprob = alg.predict_proba(dtrain[predictors])[:,1]
        
    print ("\nModel Report")
    print ("Accuracy : %.4g" % metrics.accuracy_score(dtrain['yield_class'].values, dtrain_predictions))
    print ("AUC Score (Train): %f" % metrics.roc_auc_score(dtrain['yield_class'], dtrain_predprob))
    #predictors = ['mkt_freeshares_rank', 'mmt_rank', 'roa_growth_rank']
    #list1 = alg.predict(dtest[predictors])    
    #dtest['yield_class'] = alg.predict(dtest[predictors])
    #dtest_predictions = alg.predict(dtest[predictors])
    dtest_predprob = alg.predict_proba(dtest[predictors])[:,1]
    #dtest.append('predprob')
    dtest['yield_class'] = alg.predict_proba(dtest[predictors])[:,1]
    #print (dtest)
   
    pp = [x for x in dtest.columns]
    pp.insert(0,'id')
    #print (pp)
    writer.writerow(pp)

    pp1 = [x for x in dtest.columns]
    for i in range(len(dtest_predprob)):
        tmp = []
        tmp.append(ids[i])
        for j in pp1:
            tmp.append(dtest[j][i])
        writer.writerow(tmp)
    print("\n")

      
#dataindex = '6m_3_18'
dataindex = '12m_3_18'
#dataindex = '12m_1_16'

start_date = '201601'
end_date = '201707'

estimator_start = 4
estimater_end = 8


for i in range(estimator_start, estimater_end):
    date = start_date
    while date!=end_date:
        if dataindex[0]=='6':
            train_file = 'training/'+dataindex+'/'+get_train2(date)
        else:
            train_file = 'training/'+dataindex+'/'+get_train(date)
        test_file = 'testing/'+get_test(date)
        write_file = 'prob/'+dataindex+'/param'+str(i+1-estimator_start)+'/'+get_prob(date)   
        print(train_file)
        print(test_file)
        print(write_file)

        estimator = i
        
        train = pd.read_csv(train_file)
        train = train._get_numeric_data()
        numeric_headers = list(train.columns.values)
        train = train.as_matrix()
        train = np.nan_to_num(train)
        train = pd.DataFrame(train)
        train.columns = numeric_headers

        test = pd.read_csv(test_file)
        IDs = test['id']
        #print(IDs)
        test = test._get_numeric_data()
        #print(test)
        numeric_headers1 = list(test.columns.values)
        test = test.as_matrix()
        test = np.nan_to_num(test)
        test = pd.DataFrame(test)
        test.columns = numeric_headers1

        csvfile = file(write_file,"wb")
        writer = csv.writer(csvfile)

        target = 'yield_class'
        target2 = 'predprob'
        IDcol = 'id'
        predictors = [x for x in train.columns if x not in [target, target2, IDcol]]
        #predictors2 = [x for x in test.columns if x not in [target, target2, IDcol]]
        
        
        print("search for the local best")
        param_test1 = { 'max_depth':range(3,10,1), 'min_child_weight':range(1,6,1)}
        param_test2 = { 'gamma':[i/10.0 for i in range(0,5)]}
        param_test3 = { 'subsample':[i/10.0 for i in range(6,10)], 'colsample_bytree':[i/10.0 for i in range(6,10)]}

        xgb1 = XGBClassifier( learning_rate =0.1, n_estimators=estimator, max_depth=6,
         min_child_weight=2, gamma=0, subsample=0.8, colsample_bytree=0.8,
         objective= 'binary:logistic', nthread=4, scale_pos_weight=1, seed=27)

        clf = GridSearchCV(xgb1, param_test1, cv=5, scoring='precision_macro')
            #clf.fit(X_train, y_train)
        clf.fit(train[predictors],train[target])
        print(clf.best_params_)

        max_dep = clf.best_params_['max_depth']
        child = clf.best_params_['min_child_weight']

        print("end of search one!!")

        xgb2 = XGBClassifier( learning_rate=0.1, n_estimators=estimator, max_depth = max_dep,
         min_child_weight=child, gamma=0, subsample=0.8, colsample_bytree=0.8,
         objective= 'binary:logistic', nthread=4, scale_pos_weight=1, seed=27)

        clf2 = GridSearchCV(xgb2, param_test2, cv=5, scoring='precision_macro')

        clf2.fit(train[predictors],train[target])
        print(clf2.best_params_)
        gam = clf2.best_params_['gamma']
        print("end of search two!!")

        xgb3 = XGBClassifier( learning_rate=0.1, n_estimators=estimator, max_depth = max_dep,
         min_child_weight=child, gamma=gam, subsample=0.8, colsample_bytree=0.8,
         objective= 'binary:logistic', nthread=4, scale_pos_weight=1, seed=27)

        clf3 = GridSearchCV(xgb3, param_test3, cv=5, scoring='precision_macro')
        clf3.fit(train[predictors],train[target])    
        print(clf3.best_params_)
        print("end of search three!!")

        sub = clf3.best_params_['subsample']
        bytree = clf3.best_params_['colsample_bytree']
        xgb4 = XGBClassifier( learning_rate=0.1, n_estimators=estimator, max_depth = max_dep,
         min_child_weight=child, gamma=gam, subsample=sub, colsample_bytree=bytree,
         objective= 'binary:logistic', nthread=4, scale_pos_weight=1, seed=27)

        #print(test)
        modelfit(xgb4,IDs,train,test,predictors,writer)
        csvfile.close()

        date = next_month(date)
    
